import json
import requests 
from requests_html import HTMLSession
import pandas as pd
import copy
import math
import time

def wy_data(用户输入的地区,用户输入职位):
    
    地区编码字典 = {
        '广州':'030200',
        '深圳':'040000'
    }

    url = "https://we.51job.com/api/job/search-pc"
    payload = {
            "api_key": "51job",
            "timestamp": "1687622267",
            "keyword": 用户输入职位,
            "function":"",
            "searchType": "2",
            "industry": "",
            "jobArea": 地区编码字典[用户输入的地区],
            "jobArea2": "",
            "landmark": "",
            "metro": "",
            "salary": "",
            "workYear": "",
            "degree": "",
            "companyType": "",
            "companySize": "",
            "jobType": "",
            "issueDate": "",
            "sortType": "0",
            "pageNum": "1",
            "requestId": "",
            "pageSize": "20",
            "source": "1",
            "accountId": "158566507",
            "pageCode":" sou|sou|soulb"
    }
    session = HTMLSession()
    headers = {
        'Accept':'application/json, text/plain, */*',
        'Accept-Encoding':'gzip, deflate, br',
        'Accept-Language':'zh-CN,zh;q=0.9,en;q=0.8,en-GB;q=0.7,en-US;q=0.6',
        'Account-Id':'158566507',
        'Connection':'keep-alive',
        'Cookie':'guid=6f998b5231cc3c80b7b7ace759c646a6; nsearch=jobarea%3D%26%7C%26ord_field%3D%26%7C%26recentSearch0%3D%26%7C%26recentSearch1%3D%26%7C%26recentSearch2%3D%26%7C%26recentSearch3%3D%26%7C%26recentSearch4%3D%26%7C%26collapse_expansion%3D; ps=needv%3D0; sensor=createDate%3D2019-06-14%26%7C%26identityType%3D1; Hm_lvt_1370a11171bd6f2d9b1fe98951541941=1684225757,1684828034,1685255288; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%22158566507%22%2C%22first_id%22%3A%2218823ad8c53eec-0420de7faf17634-7b515477-1327104-18823ad8c5426%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%7D%2C%22identities%22%3A%22eyIkaWRlbnRpdHlfY29va2llX2lkIjoiMTg4MjNhZDhjNTNlZWMtMDQyMGRlN2ZhZjE3NjM0LTdiNTE1NDc3LTEzMjcxMDQtMTg4MjNhZDhjNTQyNiIsIiRpZGVudGl0eV9sb2dpbl9pZCI6IjE1ODU2NjUwNyJ9%22%2C%22history_login_id%22%3A%7B%22name%22%3A%22%24identity_login_id%22%2C%22value%22%3A%22158566507%22%7D%2C%22%24device_id%22%3A%2218823ad8c53eec-0420de7faf17634-7b515477-1327104-18823ad8c5426%22%7D; partner=sem_pc360s10_10341; adv=ad_logid_url%3Dhttps%253A%252F%252Ftrace.51job.com%252Ftrace.php%253Fpartner%253Dsem_pc360s10_10341%2526ajp%253DaHR0cHM6Ly9ta3QuNTFqb2IuY29tL3RnL3NlbS9MUF8yMDIyX0JDLmh0bWw%252FZnJvbT0zNjBhZCZwYXJ0bmVyPXNlbV9wYzM2MHMxMF8xMDM0MQ%253D%253D%2526k%253D73d4113005f74cba1d7a33f1604e4354%2526qhclickid%253D1d2f24e9ddf5075c%26%7C%26; slife=lowbrowser%3Dnot%26%7C%26lastlogindate%3D20230624%26%7C%26; 51job=cuid%3D158566507%26%7C%26cusername%3DZ5M1xCHaiKLNbNSQ%252Bn3KhiZGKdiiO5w7GCN4qvuxHac%253D%26%7C%26cpassword%3D%26%7C%26cname%3DkrxcUclCCtmT%252F39bMNWvKA%253D%253D%26%7C%26cemail%3D%26%7C%26cemailstatus%3D0%26%7C%26cnickname%3D%26%7C%26ccry%3D.0PSw1jvjXmso%26%7C%26cconfirmkey%3D%25241%2524Qud%252Fv4br%25242DWFNAfX7FTmvocPOGHKo%252F%26%7C%26cautologin%3D1%26%7C%26cenglish%3D0%26%7C%26sex%3D1%26%7C%26cnamekey%3D%25241%2524PZ1bccbC%2524WxAbesuDbjFfD3lH9niB%252F1%26%7C%26to%3D901d950f6973640de8491ff64904d76664971251%26%7C%26; acw_tc=ac11000116876222255303132e00defcddc65be1836ee34f58d69b0618e317; acw_sc__v2=6497125b6cebad5e11688beed1f5e8a2d5228c7d; JSESSIONID=09923E4F01FBE0D2587B7638666A9E57; search=jobarea%7E%60%7C%21ord_field%7E%601%7C%21recentSearch0%7E%60000000%A1%FB%A1%FA000000%A1%FB%A1%FA0000%A1%FB%A1%FA00%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA9%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA0%A1%FB%A1%FA%B2%FA%C6%B7%BE%AD%C0%ED%A1%FB%A1%FA2%A1%FB%A1%FA1%7C%21recentSearch1%7E%60030200%A1%FB%A1%FA000000%A1%FB%A1%FA0000%A1%FB%A1%FA00%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA9%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA0%A1%FB%A1%FA%B2%FA%C6%B7%BE%AD%C0%ED%A1%FB%A1%FA2%A1%FB%A1%FA1%7C%21recentSearch2%7E%60040000%A1%FB%A1%FA000000%A1%FB%A1%FA0000%A1%FB%A1%FA00%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA9%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA0%A1%FB%A1%FA%B2%FA%C6%B7%BE%AD%C0%ED%A1%FB%A1%FA2%A1%FB%A1%FA1%7C%21recentSearch3%7E%60030200%2C040000%A1%FB%A1%FA000000%A1%FB%A1%FA0000%A1%FB%A1%FA00%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA9%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA0%A1%FB%A1%FA%B2%FA%C6%B7%BE%AD%C0%ED%A1%FB%A1%FA2%A1%FB%A1%FA1%7C%21recentSearch4%7E%60000000%A1%FB%A1%FA000000%A1%FB%A1%FA0000%A1%FB%A1%FA00%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA99%A1%FB%A1%FA9%A1%FB%A1%FA99%A1%FB%A1%FA%A1%FB%A1%FA0%A1%FB%A1%FA%D0%C2%C3%BD%CC%E5%D4%CB%D3%AA%A1%FB%A1%FA2%A1%FB%A1%FA1%7C%21collapse_expansion%7E%601%7C%21; ssxmod_itna=eqUxR7DQDt0Q74BPGIrCDjOr0AKYvvEEAXpdD/KDfo4iNDnD8x7YDv+mvFmWQFOiDP0KHFcDqfP7IG4q5We8QDa+fH+DB3DEx065rtxYYkDt4DTD34DYDixib1xi5GRD0KDFF5XUZ9Dm4GWFqGfDDoDY86RDitD4qDBGOdDKqGgFq267mt3pLxe57GcD0tdxBLat=hcGeaaTiNexanDrEQDzqHDtutS9Ld3x0PyBMUDw0GIWt+ztOPaI8hNIYxmY2DQbYxef9xwKBhNSiqjm0+eDDfN82GdeD===; ssxmod_itna2=eqUxR7DQDt0Q74BPGIrCDjOr0AKYvvEEAXPG9t5CDfxGNj7GafKmHkzx82iIVzGd1=lck5dsTmZiBqsLAoCiN17m6ifa1dD3nDr1smr0dxsC2tQ2OOseRbMDmK4Rwfc1qeLTEk1RdS3EAfvxbifc/o=U/2XzFd65pxhSmewIIj=XRnNkj2N+Y3=LWRihQujnQu3VeuQVQToapxoTcoFxcxo+lTorpSK=VO=5qt=kFuGYph7GHwQ/4998u6Whr4RyBlnGh6khj3M67F7U8zluqNHuP9ju2/8RdFC+FuFpWCuVeI3ktIlrDjZgIegX=1tkQiQE0sjXV2imEE4zw7B+SoYogIdSKpnQ9gKchPqoNH0tT=PRoRQS+xnr=cRWRbBgK7I6QibDcjC/bcrTH3bB1Rd3R=RbooRo0UKmWFIGdbGTDG2WIC83Pgbq0DrASdKG=7xxFTFMr5bDDFqD+pDxD===',
        'From-Domain':'51job_web',
        'Host':'we.51job.com',
        'Partner':'sem_pc360s10_10341',
        'Property':'%7B%22partner%22%3A%22sem_pc360s10_10341%22%2C%22webId%22%3A2%2C%22fromdomain%22%3A%2251job_web%22%2C%22frompageUrl%22%3A%22https%3A%2F%2Fwe.51job.com%2F%22%2C%22pageUrl%22%3A%22https%3A%2F%2Fwe.51job.com%2Fpc%2Fsearch%3Fkeyword%3D%25E4%25BA%25A7%25E5%2593%2581%25E7%25BB%258F%25E7%2590%2586%22%2C%22identityType%22%3A%22%E8%81%8C%E5%9C%BA%E4%BA%BA%22%2C%22userType%22%3A%22%E8%80%81%E7%94%A8%E6%88%B7%22%2C%22isLogin%22%3A%22%E6%98%AF%22%2C%22accountid%22%3A%22158566507%22%7D',
        'Referer':'https://we.51job.com/pc/search?keyword=%E4%BA%A7%E5%93%81%E7%BB%8F%E7%90%86',
        'Sec-Ch-Ua':'"Not.A/Brand";v="8", "Chromium";v="114", "Microsoft Edge";v="114"',
        'Sec-Ch-Ua-Mobile':'?0',
        'Sec-Ch-Ua-Platform':"Windows",
        'Sec-Fetch-Dest':'empty',
        'Sec-Fetch-Mode':'cors',
        'Sec-Fetch-Site':'same-origin',
        'Sign':'7173106cbd3975d183269858ce261449c8dca423266505914168504d7a3c8248',
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36 Edg/114.0.1823.51',
        'User-Token':'6314673d0f7c50682646452dfe9403ae6496c5b9',
        'Uuid':'6f998b5231cc3c80b7b7ace759c646a6',
        }
    r = session.get(url=url,params=payload,headers=headers)
    response_data = r.json()
    page = math.ceil(int(response_data['resultbody']['job']['totalCount'])/int(payload['pageSize']))
    response_df = []
    for i in range(page): # 需要判断页面的数据有多少页
        payload['pageNum']=i
        # send a POST request with headers
        r = session.get(url=url,params=payload,headers=headers)
        # extract the JSON data from the response
        response_data = r.json()
        df = pd.json_normalize(response_data['resultbody']['job']['items'])
        response_df.append(df)
    response_df
    # 整理表格并输出数据
    df = pd.concat(response_df)
    keyword = payload['keyword']
    output_time = str(time.localtime().tm_mon)\
             +str(time.localtime().tm_mday)+'_'\
             +str(time.localtime().tm_hour) \
             +str(time.localtime().tm_min)
    df.to_excel( keyword +'_51job_'+output_time+'.xlsx')
    return "当前数据已导出，数据量为：",len(df),"行"

def wy_dq(用户输入的地区, 用户输入职位):
    """ 猎聘数据 地区分布 数据分析可视化 """
    output_time = str(time.localtime().tm_mon)\
         +str(time.localtime().tm_mday)+'_'\
         +str(time.localtime().tm_hour) \
         +str(time.localtime().tm_min)
    # 数据爬取
    df = pd.read_excel(f'{用户输入职位}_wy_{output_time}.xlsx')
    df_data = df[['companyTypeString','jobTags','jobAreaString','provideSalaryString','termStr','companyName','degreeString','industryType1Str','companySizeString']]

    
    # 数据分析
    dq = [ i.split('·')[1] for i in df_data['jobAreaString'].value_counts().index.tolist() if '·'  in i]
    gw= df_data['jobAreaString'].value_counts().values.tolist()
    
    # 数据可视化
    c = (
        Map()
        .add(用户输入的地区, [list(z) for z in zip(dq, gw)], 用户输入的地区)
        .set_global_opts(
            title_opts=opts.TitleOpts(title='Map-'+用户输入的地区+'地图'), visualmap_opts=opts.VisualMapOpts()
        )
    )
    return c